<!DOCTYPE html>

<html lang="en">
  <head>
    <meta charset="utf-8" />
    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
    <title>cup.services.buffers &#8212; cup 3.2.33 documentation</title>
    <link rel="stylesheet" type="text/css" href="../../../_static/pygments.css" />
    <link rel="stylesheet" type="text/css" href="../../../_static/alabaster.css" />
    <script data-url_root="../../../" id="documentation_options" src="../../../_static/documentation_options.js"></script>
    <script src="../../../_static/doctools.js"></script>
    <script src="../../../_static/sphinx_highlight.js"></script>
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" />
   
  <link rel="stylesheet" href="../../../_static/custom.css" type="text/css" />
  
  
  <meta name="viewport" content="width=device-width, initial-scale=0.9, maximum-scale=0.9" />

  </head><body>
  

    <div class="document">
      <div class="documentwrapper">
        <div class="bodywrapper">
          

          <div class="body" role="main">
            
  <h1>Source code for cup.services.buffers</h1><div class="highlight"><pre>
<span></span><span class="ch">#!/usr/bin/env python</span>
<span class="c1"># -*- coding: utf-8 -*</span>
<span class="c1"># Copyright: [CUP] - See LICENSE for details.</span>
<span class="c1"># Authors: Guannan Ma (@mythmgn),</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">:description:</span>
<span class="sd">    buffer pool</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="kn">import</span> <span class="nn">threading</span>

<span class="kn">from</span> <span class="nn">cup</span> <span class="kn">import</span> <span class="n">platforms</span>

<span class="n">SMALL_BLOCK_SIZE</span> <span class="o">=</span> <span class="mi">4096</span> <span class="c1"># 4kb</span>
<span class="n">MEDIUM_BLOCK_SIZE</span> <span class="o">=</span> <span class="p">(</span><span class="mi">128</span> <span class="o">+</span> <span class="mi">4</span><span class="p">)</span> <span class="o">*</span> <span class="mi">1024</span>   <span class="c1"># (128 + 4) K</span>
<span class="n">LARGE_BLOCK_SIZE</span> <span class="o">=</span> <span class="mi">1</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span> <span class="o">+</span> <span class="n">SMALL_BLOCK_SIZE</span>  <span class="c1"># 1M + 4K</span>

<span class="n">SMALL_BLOCK</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">MEDIUM_BLOCK</span> <span class="o">=</span> <span class="mi">1</span>
<span class="n">LARGE_BLOCK</span> <span class="o">=</span> <span class="mi">2</span>


<div class="viewcode-block" id="Buffer"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer">[docs]</a><span class="k">class</span> <span class="nc">Buffer</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Buffer object which you get from BufferPool.allocate(num).</span>

<span class="sd">    A **Buffer** consists of several bytearrays which is mutable compared to</span>
<span class="sd">    a normal **str**. In other words, if you have senarios like: allocte mem</span>
<span class="sd">    and deallocte mem frenquently. With high chance you can replace directly</span>
<span class="sd">    using str by Buffer. It will reduce the memory fragments.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">items</span><span class="p">,</span> <span class="n">block_size</span><span class="p">,</span> <span class="n">uniqid</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_items</span> <span class="o">=</span> <span class="n">items</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span> <span class="o">=</span> <span class="n">block_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_num</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">items</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_length</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_uniqid</span> <span class="o">=</span> <span class="n">uniqid</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_maxsize</span> <span class="o">=</span> <span class="n">block_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num</span>

<div class="viewcode-block" id="Buffer.set"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.set">[docs]</a>    <span class="k">def</span> <span class="nf">set</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">content</span><span class="p">,</span> <span class="n">encoding</span><span class="o">=</span><span class="s1">&#39;utf8&#39;</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        set content to buffers</span>

<span class="sd">        :return:</span>
<span class="sd">            return (True, None) if succeed.</span>
<span class="sd">            return (False, error_msg) otherwise</span>

<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">platforms</span><span class="o">.</span><span class="n">is_py3</span><span class="p">():</span>
            <span class="n">content</span> <span class="o">=</span> <span class="n">content</span><span class="o">.</span><span class="n">encode</span><span class="p">(</span><span class="n">encoding</span><span class="p">)</span>
        <span class="n">length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">content</span><span class="p">)</span>
        <span class="n">ind</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="n">item_ind</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">if</span> <span class="n">length</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">_num</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">(</span><span class="kc">False</span><span class="p">,</span> <span class="s1">&#39;content size &gt; Buffer size&#39;</span><span class="p">)</span>
        <span class="k">while</span> <span class="n">ind</span> <span class="o">&lt;</span> <span class="n">length</span><span class="p">:</span>
            <span class="k">if</span> <span class="p">(</span><span class="n">ind</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span><span class="p">)</span> <span class="o">&lt;=</span> <span class="p">(</span><span class="n">length</span> <span class="o">-</span> <span class="mi">1</span><span class="p">):</span>
                <span class="n">loop_size</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">loop_size</span> <span class="o">=</span> <span class="n">length</span> <span class="o">-</span> <span class="n">ind</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_items</span><span class="p">[</span><span class="n">item_ind</span><span class="p">]</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">content</span><span class="p">[</span><span class="n">ind</span><span class="p">:</span> <span class="n">loop_size</span><span class="p">])</span>
            <span class="n">item_ind</span> <span class="o">+=</span> <span class="mi">1</span>
            <span class="n">ind</span> <span class="o">+=</span> <span class="n">loop_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_length</span> <span class="o">=</span> <span class="n">length</span>
        <span class="k">return</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span></div>

<div class="viewcode-block" id="Buffer.get"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.get">[docs]</a>    <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        return (True, (blocks, block_size, total_length, encoding)) if succeed</span>

<span class="sd">        Otherwise, return (False, err_msg, None)</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">rev</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_length</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">rev</span> <span class="o">=</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_items</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_length</span><span class="p">))</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">rev</span> <span class="o">=</span> <span class="p">(</span><span class="kc">False</span><span class="p">,</span> <span class="p">(</span><span class="s1">&#39;not initialized yet&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span><span class="p">,</span> <span class="kc">None</span><span class="p">))</span>
        <span class="k">return</span> <span class="n">rev</span></div>

<div class="viewcode-block" id="Buffer.get_uniq_id"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.get_uniq_id">[docs]</a>    <span class="k">def</span> <span class="nf">get_uniq_id</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        return the uniqid for this object</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_uniqid</span></div>

<div class="viewcode-block" id="Buffer.get_byte_arrays"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.get_byte_arrays">[docs]</a>    <span class="k">def</span> <span class="nf">get_byte_arrays</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        get byte arrays in the buffer</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_items</span></div>

<div class="viewcode-block" id="Buffer.maxsize"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.maxsize">[docs]</a>    <span class="k">def</span> <span class="nf">maxsize</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;return how many unicode/str you can set to the buffer&quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_maxsize</span></div>

<div class="viewcode-block" id="Buffer.length"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.Buffer.length">[docs]</a>    <span class="k">def</span> <span class="nf">length</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;return the length you have used for the buffer&quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_length</span></div></div>


<span class="c1"># pylint: disable=R0902</span>
<div class="viewcode-block" id="BufferPool"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.BufferPool">[docs]</a><span class="k">class</span> <span class="nc">BufferPool</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
<span class="w">    </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Buffer pool class which will ease memory fragment.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
            <span class="bp">self</span><span class="p">,</span> <span class="n">pool_size</span><span class="p">,</span> <span class="n">block_size</span><span class="o">=</span><span class="n">MEDIUM_BLOCK_SIZE</span><span class="p">,</span> <span class="n">extendable</span><span class="o">=</span><span class="kc">False</span>
    <span class="p">):</span>
        <span class="k">if</span> <span class="n">block_size</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span>
                <span class="n">SMALL_BLOCK_SIZE</span><span class="p">,</span> <span class="n">MEDIUM_BLOCK_SIZE</span><span class="p">,</span> <span class="n">LARGE_BLOCK_SIZE</span>
        <span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
                <span class="s1">&#39;block_size should be buffers.SMALL_BLOCK_SIZE&#39;</span>
                <span class="s1">&#39; or buffers.MEDIUM_BLOCK_SIZE or buffers.LARGE_BLOCK_SIZE&#39;</span>
            <span class="p">)</span>
        <span class="c1"># TODO If extendable, we should expand the pool</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_extendable</span> <span class="o">=</span> <span class="n">extendable</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_free_list</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_used_dict</span> <span class="o">=</span> <span class="p">{}</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_pool_size</span> <span class="o">=</span> <span class="n">pool_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_used_num</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pool_size</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">_used_num</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span> <span class="o">=</span> <span class="n">block_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_uniqid</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span> <span class="o">=</span> <span class="n">block_size</span>
        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_pool_size</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_free_list</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="nb">bytearray</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span><span class="p">))</span>

<div class="viewcode-block" id="BufferPool.allocate"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.BufferPool.allocate">[docs]</a>    <span class="k">def</span> <span class="nf">allocate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">num</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        acclocate buff with num * block_size</span>

<span class="sd">        :return:</span>
<span class="sd">            (True, Buffer object)</span>

<span class="sd">            (False, str_error_msg)</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">ret</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">acquire</span><span class="p">()</span>
        <span class="k">if</span> <span class="n">num</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span><span class="p">:</span>
            <span class="n">ret</span> <span class="o">=</span> <span class="p">(</span><span class="kc">False</span><span class="p">,</span> <span class="s1">&#39;not enough free buffer available&#39;</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">uniqid</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">current_thread</span><span class="p">()</span><span class="o">.</span><span class="n">ident</span>
            <span class="n">ind</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span> <span class="o">-</span> <span class="n">num</span>
            <span class="n">buff</span> <span class="o">=</span> <span class="n">Buffer</span><span class="p">(</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_free_list</span><span class="p">[</span><span class="n">ind</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span><span class="p">],</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_block_size</span><span class="p">,</span>
                <span class="n">uniqid</span>
            <span class="p">)</span>
            <span class="n">ret</span> <span class="o">=</span> <span class="p">(</span><span class="kc">True</span><span class="p">,</span> <span class="n">buff</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_used_dict</span><span class="p">[</span><span class="n">uniqid</span><span class="p">]</span> <span class="o">=</span> <span class="n">ret</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span>
            <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_free_list</span><span class="p">[</span><span class="n">ind</span><span class="p">:</span> <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span><span class="p">]</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span> <span class="o">-=</span> <span class="n">num</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_used_num</span> <span class="o">+=</span> <span class="n">num</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">ret</span></div>

<div class="viewcode-block" id="BufferPool.deallocate"><a class="viewcode-back" href="../../../cup.services.html#cup.services.buffers.BufferPool.deallocate">[docs]</a>    <span class="k">def</span> <span class="nf">deallocate</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">buff</span><span class="p">):</span>
<span class="w">        </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        return the acclocated buff back to the pool</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="n">buff</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">return</span> <span class="kc">False</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">acquire</span><span class="p">()</span>
        <span class="n">uniqid</span> <span class="o">=</span> <span class="n">buff</span><span class="o">.</span><span class="n">get_uniq_id</span><span class="p">()</span>
        <span class="k">if</span> <span class="n">uniqid</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_used_dict</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;this buff is not in the pool!!!&#39;</span><span class="p">)</span>
        <span class="n">byte_arrays</span> <span class="o">=</span> <span class="n">buff</span><span class="o">.</span><span class="n">get_byte_arrays</span><span class="p">()</span>
        <span class="n">length</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">byte_arrays</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_free_list</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">byte_arrays</span><span class="p">)</span>
        <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">_used_dict</span><span class="p">[</span><span class="n">uniqid</span><span class="p">]</span>
        <span class="k">del</span> <span class="n">buff</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_used_num</span> <span class="o">-=</span> <span class="n">length</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_free_num</span> <span class="o">+=</span> <span class="n">length</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_lock</span><span class="o">.</span><span class="n">release</span><span class="p">()</span>
        <span class="k">return</span> <span class="kc">True</span></div></div>

<span class="c1"># vi:set tw=0 ts=4 sw=4 nowrap fdm=indent</span>
</pre></div>

          </div>
          
        </div>
      </div>
      <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
        <div class="sphinxsidebarwrapper">
<h1 class="logo"><a href="../../../index.html">cup</a></h1>








<h3>Navigation</h3>
<p class="caption" role="heading"><span class="caption-text">Contents:</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../cup.html">cup package</a></li>
</ul>

<div class="relations">
<h3>Related Topics</h3>
<ul>
  <li><a href="../../../index.html">Documentation overview</a><ul>
  <li><a href="../../index.html">Module code</a><ul>
  </ul></li>
  </ul></li>
</ul>
</div>
<div id="searchbox" style="display: none" role="search">
  <h3 id="searchlabel">Quick search</h3>
    <div class="searchformwrapper">
    <form class="search" action="../../../search.html" method="get">
      <input type="text" name="q" aria-labelledby="searchlabel" autocomplete="off" autocorrect="off" autocapitalize="off" spellcheck="false"/>
      <input type="submit" value="Go" />
    </form>
    </div>
</div>
<script>document.getElementById('searchbox').style.display = "block"</script>








        </div>
      </div>
      <div class="clearer"></div>
    </div>
    <div class="footer">
      &copy;2023, CUP-DEV Team.
      
      |
      Powered by <a href="http://sphinx-doc.org/">Sphinx 7.0.1</a>
      &amp; <a href="https://github.com/bitprophet/alabaster">Alabaster 0.7.13</a>
      
    </div>

    

    
  </body>
</html>